package basic;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class Demo2 {

	/**简单数据去重
	 * 使用的是shuffle自带的去重排序功能
	 * @param args
	 */
	public static void main(String[] args) throws Exception {
		// TODO Auto-generated method stub
		if (args.length!=2) {
			System.err.println("user infos err: <inpath>,<outpath>");
			System.exit(-1);
		}
		@SuppressWarnings("deprecation")
		Job job = new Job(new Configuration(), "quchong");
		job.setJarByClass(Demo2.class);
		
		FileInputFormat.addInputPath(job, new Path(args[0]));//输入路径
		FileOutputFormat.setOutputPath(job,new Path(args[1]));//输出路径
		
		job.setMapperClass(qMap.class);
		job.setReducerClass(qReduce.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(NullWritable.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		job.waitForCompletion(true);

	}
	public static class qMap extends Mapper<LongWritable, Text, Text, NullWritable>{
		protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,Text,NullWritable>.Context context) throws java.io.IOException ,InterruptedException {
		context.write(value,NullWritable.get());
		};
	}
	//map结果会通过shuffle阶段处理，会将相同key的value放在同一个value集合中，从而实现对key的去重。去重后的结果，会交给reduce函数处理
	public static class qReduce extends Reducer<Text, IntWritable, Text, NullWritable>{
		protected void reduce(Text k2, java.lang.Iterable<IntWritable> value, org.apache.hadoop.mapreduce.Reducer<Text,IntWritable,Text,NullWritable>.Context context) throws java.io.IOException ,InterruptedException {
			 context.write(k2,NullWritable.get());
		};
	}

}
